package com.mz.webmagic.processor;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * 
 * Mzqingfeng
 * 爬虫核心类
 * 
 */
public class OsChinaProcessor implements PageProcessor {
	/**
	 * 抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	 */
	private Site site = Site.me().setRetryTimes(3).setSleepTime(100);

	@Override
	public Site getSite() {
		return site;
	}

	/**
	 * process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
	 */
	@Override
	public void process(Page page) {
		
	}
	
	public static void main(String[] args) {
        long startTime, endTime;
        
        System.out.println("开始爬取...");
        startTime = System.currentTimeMillis();
        
        Spider.create(new OsChinaProcessor())
        .addUrl("https://www.oschina.net/project/zh")
        .thread(5)
        .run();
        
        /*Spider spider = Spider.create(new OsChinaProcessor());
        spider.addUrl("http://aj.china.com.cn");
        spider.addPipeline(new OsChinaPipeline());
        spider.thread(5);
        spider.setExitWhenComplete(true);
        spider.start();
        spider.stop();*/
        
        endTime = System.currentTimeMillis();
    }

}
